split_graph(s1, s2)
def forward(self, x1: float, x2: float) -> float:
return self.w1 * x1 + self.w2 * x2 + self.b
@dataclass
class Linear:
# Parameters
w1: float
w2: float
b: float
def forward(self, x1: float, x2: float) -> float:
return self.w1 * x1 + self.w2 * x2 + self.b
model = Linear(1, 1, -0.9)
draw_graph(model)
with_points(s1, s2, Linear(1, 1, -0.4))
def point_loss(x):
return -math.log(minitorch.operators.sigmoid(-x))
graph(point_loss, [], [])
graph(point_loss, [], [-2, -0.2, 1])
hcat([show(Linear(1, 1, -0.6)),
show(Linear(1, 1, -0.7)),
show(Linear(1, 1, -0.8))], 0.3)
model1 = Linear(1, 1, -0.4)
model2 = Linear(1, 1, -0.5)
compare(model1, model2)
with_points(s1, s2, Linear(1, 1, -1.5))
def point_loss(out, y=1):
return (y * # Correct Side
-math.log(minitorch.operators.sigmoid( # Log-Sigmoid
-out))) # Distance
def full_loss(m):
l = 0
for x, y in zip(s.X, s.y):
l += point_loss(-m.forward(*x), y)
return -l
hcat([graph(point_loss, [], [-2, -0.2, 1]),
graph(lambda x: point_loss(-x), [-1, 0.4, 1.3], [])], 0.3)
hcat([show(Linear(1, 1, -1.5)),
show(Linear(1, 1, -1.45))], 0.3)
set_svg_height(300)
show_loss(full_loss, Linear(1, 1, 0))
How do we find the right direction?
How small changes in input impact output.
$$f(x) = x^2 + 1$$
def f(x):
return x * x + 1.0
plot_function("f(x)", f)
$$f(x) = x^2 + 1$$ $$f'(x) = 2x$$
def f_prime(x):
return 2 * x
def tangent_line(slope, x, y):
def line(x_):
return slope * (x_ - x) + y
return line
plot_function("f(x) vs f'(2)",
f, fn2=tangent_line(f_prime(2), 2, f(2)))
$$f(x) = \sin(2 x)$$
plot_function("f(x) = sin(2x)", lambda x: math.sin(2 * x))
$$f(x) = \sin(2 x) \Rightarrow f'(x) = 2 \cos(2 x)$$
plot_function("f'(x) = 2*cos(2x)", lambda x: 2 * math.cos(2 * x),
fn2=lambda x: math.sin(2 * x))
$$f(x, y) = \sin(x) + \cos(y)$$
plot_function3D("f(x, y) = sin(x) + 2 * cos(y)", lambda x,y: math.sin(x) + 2 * math.cos(y))
$$f_x'(x, y) = \cos(x) \ \ \ f_y'(x, y) = -2 \sin(y)$$
plot_function3D("f'_x(x, y) = cos(x)", lambda x, y: math.cos(x))
def f(x: float) -> float:
...
$$f(x) = ...$$ $$f'(x) = ...$$
def derivative(f: Callable[[float], float]
) -> Callable[[float], float]:
def f_prime(x: float) -> float:
...
return f_prime
$$f'(x) = \lim_{\epsilon \rightarrow 0} \frac{f(x + \epsilon) - f(x - \epsilon)}{2\epsilon}$$
Approximate derivative
$$f'(x) \approx \frac{f(x + \epsilon) - f(x-\epsilon)}{2\epsilon}$$

Key Idea: Only need to call $f$.
def central_difference(f: Callable[[float], float],
x: float) -> float:
...
$$f(x) = ...$$ $$f'(x) = ...$$
def derivative(f: Callable[[float], float]
) -> Callable[[float], float]:
def f_prime(x: float) -> float:
return minitorch.central_difference(f, x)
return f_prime
Turn 2-argument function into 1-arg.
def f(x, y):
...
def f_x_prime(x: float, y: float) -> float:
def inner(x: float) -> float:
return f(x, y)
return derivative(inner)(x)
def sigmoid(x: float) -> float:
if x >= 0:
return 1.0 / (1.0 + math.exp(-x))
else:
return math.exp(x) / (1.0 + math.exp(x))
plot_function("sigmoid", sigmoid)
sigmoid_prime = derivative(sigmoid)
plot_function("Derivative of sigmoid", sigmoid_prime)